{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 推荐系统 模型验证指标\n",
    "[MAP](#MAP)   Mean Average Precision at k   平均准确率  \n",
    "[NDCG](#NDCG)   Normalized Discounted Cumulative Gain (归一化折损累计增益)    \n",
    "[Precision](#Precision)   \n",
    "[Recall](#Recall)    \n",
    "[RMSE](#RMSE)  Root Mean Squared Error   均方根误差  \n",
    "[MAE](#MAE)    Mean Absolute Error 平均绝对误差  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Platform : win32 [win32/linux]\n",
      "Systerm  : 3.6.10 (default, Mar  5 2020, 10:17:47) [MSC v.1900 64 bit (AMD64)] \n",
      "numpy  Version: 1.16.0\n",
      "pandas Version: 1.0.4\n",
      "sklearn Version: 0.23.2\n"
     ]
    }
   ],
   "source": [
    "# set the environment path to find Recommenders\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "\n",
    "from common.python_evaluation import (\n",
    "    map_at_k,\n",
    "    ndcg_at_k,\n",
    "    precision_at_k,\n",
    "    recall_at_k,\n",
    "    rmse,\n",
    "    mae,\n",
    "    logloss,\n",
    "    rsquared,\n",
    "    exp_var\n",
    ")\n",
    "\n",
    "from sklearn.metrics import (\n",
    "    mean_squared_error,\n",
    "    mean_absolute_error,\n",
    "    r2_score,\n",
    "    explained_variance_score,\n",
    "    roc_auc_score,\n",
    "    log_loss,\n",
    ")\n",
    "\n",
    "print('Platform : {} [win32/linux]'.format(sys.platform))  # 当前平台信息 \n",
    "print('Systerm  : {} '.format(sys.version))\n",
    "print('numpy  Version: {}'.format(np.__version__))\n",
    "print('pandas Version: {}'.format(pd.__version__))\n",
    "print('sklearn Version: {}'.format(sklearn.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userID</th>\n",
       "      <th>itemID</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userID  itemID  rating\n",
       "0       1       6     0.9\n",
       "1       1       2     0.5\n",
       "2       1       3     1.0\n",
       "3       1       4     1.0\n",
       "4       1       5     0.5\n",
       "5       1       1     0.5"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y1_true = pd.DataFrame({'userID': [1, 1, 1, 1, 1, 1],\n",
    "                       'itemID': [6, 2, 3, 4, 5, 1],\n",
    "                       'rating': [0.9, 0.5, 1.0, 1.0, 0.5, 0.5]})\n",
    "\n",
    "y1_true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userID</th>\n",
       "      <th>itemID</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userID  itemID  rating\n",
       "0       2       5     1.0\n",
       "1       2       3     0.8\n",
       "2       2       1     0.9\n",
       "3       2       4     0.5"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y2_true = pd.DataFrame({'userID': [2, 2, 2, 2],\n",
    "                       'itemID': [5, 3, 1, 4],\n",
    "                       'rating': [1.0, 0.8, 0.9, 0.5]})\n",
    "\n",
    "y2_true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userID</th>\n",
       "      <th>itemID</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>0.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.50</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userID  itemID  prediction\n",
       "1       1       3        1.00\n",
       "2       1      20        0.97\n",
       "0       1       1        0.50"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prediction 用于 item 排序  名称不可修改\n",
    "y1_pre = pd.DataFrame({'userID': [1, 1, 1],\n",
    "                      'itemID': [1, 3, 20],\n",
    "                      'prediction': [0.5, 1.0, 0.97]})\n",
    "\n",
    "# 按 userID prediction 降序\n",
    "y1_pre = y1_pre.sort_values(['userID', 'prediction'], axis=0, ascending=False)\n",
    "y1_pre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userID</th>\n",
       "      <th>itemID</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>0.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0.92</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userID  itemID  prediction\n",
       "1       2       3        1.00\n",
       "2       2       1        0.97\n",
       "3       2       4        0.97\n",
       "0       2       5        0.92"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prediction 用于 item 排序\n",
    "y2_pre = pd.DataFrame({'userID': [2, 2, 2, 2],\n",
    "                      'itemID': [5, 3, 1, 4],\n",
    "                      'prediction': [0.92, 1.0, 0.97, 0.97]})\n",
    "\n",
    "# 按 userID prediction 降序\n",
    "y2_pre = y2_pre.sort_values(['userID', 'prediction'], axis=0, ascending=False)\n",
    "y2_pre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userID</th>\n",
       "      <th>itemID</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userID  itemID  rating\n",
       "0       1       6     0.9\n",
       "1       1       2     0.5\n",
       "2       1       3     1.0\n",
       "3       1       4     1.0\n",
       "4       1       5     0.5\n",
       "5       1       1     0.5\n",
       "0       2       5     1.0\n",
       "1       2       3     0.8\n",
       "2       2       1     0.9\n",
       "3       2       4     0.5"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_true = pd.concat([y1_true, y2_true])\n",
    "y_true"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userID</th>\n",
       "      <th>itemID</th>\n",
       "      <th>prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>0.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>0.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>0.92</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userID  itemID  prediction\n",
       "1       1       3        1.00\n",
       "2       1      20        0.97\n",
       "0       1       1        0.50\n",
       "1       2       3        1.00\n",
       "2       2       1        0.97\n",
       "3       2       4        0.97\n",
       "0       2       5        0.92"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pre = pd.concat([y1_pre, y2_pre])\n",
    "y_pre"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "--- \n",
    "##  MAP  \n",
    "Mean Average Precision at k  \n",
    "http://web.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf   \n",
    "MAP(Mean Average Precision)：单个主题的平均准确率是每篇相关文档检索出后的准确率的平均值。 \n",
    "\n",
    "主集合的平均准确率(MAP)是每个主题的平均准确率的平均值。\n",
    "\n",
    "MAP 是反映系统在全部相关文档上性能的单值指标。\n",
    "\n",
    "系统检索出来的相关文档越靠前(rank 越高)，MAP就可能越高。如果系统没有返回相关文档，则准确率默认为0。\n",
    "例如：\n",
    "\n",
    "假设有两个主题，主题1有4个相关网页，主题2有5个相关网页。\n",
    "\n",
    "某系统对于主题1检索出4个相关网页，其rank分别为1, 2, 4, 7；\n",
    "\n",
    "对于主题2检索出3个相关网页，其rank分别为1,3,5。\n",
    "\n",
    "对于主题1，平均准确率为(1/1+2/2+3/4+4/7)/4=0.83。对于主题2，平均准确率为(1/1+2/3+3/5+0+0)/5=0.45。\n",
    "\n",
    "则MAP= (0.83+0.45)/2=0.64。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y1_true  [6, 2, 3, 4, 5, 1]\n",
      "y1_pre   [3, 20, 1]\n",
      "TopK 1 :0.167\n",
      "TopK 2 :0.167\n",
      "TopK 3 :0.278\n",
      "TopK 4 :0.278\n",
      "TopK 5 :0.278\n",
      "TopK 6 :0.278\n",
      "\n",
      "TopK 1: (1/1)/6 =0.167\n",
      "TopK 2: (1/1+0/2)/6 =0.167\n",
      "TopK 3: (1/1+0/2+2/3)/6 =0.278\n",
      "TopK 4: (1/1+0/2+2/3+0)/6 =0.278\n",
      "TopK 5: (1/1+0/2+2/3+0+0)/6 =0.278\n"
     ]
    }
   ],
   "source": [
    "print('y1_true ', y1_true['itemID'].tolist())\n",
    "print('y1_pre  ', y1_pre['itemID'].tolist())\n",
    "\n",
    "eval_1_map = list()\n",
    "for top_k in range(1, 7):\n",
    "    # GroundTruth 中的 col_rating 在此不起任何作用\n",
    "    eval_map = map_at_k(y1_true, y1_pre, col_user='userID', col_item='itemID', col_rating='userID', k=top_k)\n",
    "    eval_1_map.append(eval_map)\n",
    "    print('TopK {} :{:.3f}'.format(top_k, eval_map))\n",
    "\n",
    "print('')\n",
    "print('TopK 1: (1/1)/6 ={:.3f}'.format((1/1)/6))\n",
    "print('TopK 2: (1/1+0/2)/6 ={:.3f}'.format((1/1+0/2)/6))\n",
    "print('TopK 3: (1/1+0/2+2/3)/6 ={:.3f}'.format((1/1+0/2+2/3)/6))\n",
    "print('TopK 4: (1/1+0/2+2/3+0)/6 ={:.3f}'.format((1/1+0/2+2/3+0)/6))\n",
    "print('TopK 5: (1/1+0/2+2/3+0+0)/6 ={:.3f}'.format((1/1+0/2+2/3+0+0)/6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y2_true  [5, 3, 1, 4]\n",
      "y2_pre   [3, 1, 4, 5]\n",
      "TopK 1 :0.250\n",
      "TopK 2 :0.500\n",
      "TopK 3 :0.750\n",
      "TopK 4 :1.000\n",
      "TopK 5 :1.000\n",
      "TopK 6 :1.000\n",
      "\n",
      "TopK 1: (1/1)/4 =0.250\n",
      "TopK 2: (1/1+2/2)/4 =0.500\n",
      "TopK 3: (1/1+2/2+3/3)/4 =0.750\n",
      "TopK 4: (1/1+2/2+3/3+4/4)/4 =1.000\n",
      "TopK 5: (1/1+2/2+3/3+4/4+0)/4 =1.000\n"
     ]
    }
   ],
   "source": [
    "print('y2_true ', y2_true['itemID'].tolist())\n",
    "print('y2_pre  ', y2_pre['itemID'].tolist())\n",
    "\n",
    "eval_2_map = list()\n",
    "for top_k in range(1, 7):\n",
    "    # GroundTruth 中的 col_rating 在此不起任何作用    \n",
    "    eval_map = map_at_k(y2_true, y2_pre, col_user='userID', col_item='itemID', col_rating='userID', k=top_k)\n",
    "    eval_2_map.append(eval_map)\n",
    "    print('TopK {} :{:.3f}'.format(top_k, eval_map))\n",
    "\n",
    "print('')\n",
    "print('TopK 1: (1/1)/4 ={:.3f}'.format((1/1)/4))\n",
    "print('TopK 2: (1/1+2/2)/4 ={:.3f}'.format((1/1+2/2)/4))\n",
    "print('TopK 3: (1/1+2/2+3/3)/4 ={:.3f}'.format((1/1+2/2+3/3)/4))\n",
    "print('TopK 4: (1/1+2/2+3/3+4/4)/4 ={:.3f}'.format((1/1+2/2+3/3+4/4)/4))\n",
    "print('TopK 5: (1/1+2/2+3/3+4/4+0)/4 ={:.3f}'.format((1/1+2/2+3/3+4/4+0)/4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_true  [6, 2, 3, 4, 5, 1, 5, 3, 1, 4]\n",
      "y_pre   [3, 20, 1, 3, 1, 4, 5]\n",
      "TopK 1 :0.208   U1: 0.167  U2: 0.250  (U1+U2)/2: 0.208\n",
      "TopK 2 :0.333   U1: 0.167  U2: 0.500  (U1+U2)/2: 0.333\n",
      "TopK 3 :0.514   U1: 0.278  U2: 0.750  (U1+U2)/2: 0.514\n",
      "TopK 4 :0.639   U1: 0.278  U2: 1.000  (U1+U2)/2: 0.639\n",
      "TopK 5 :0.639   U1: 0.278  U2: 1.000  (U1+U2)/2: 0.639\n",
      "TopK 6 :0.639   U1: 0.278  U2: 1.000  (U1+U2)/2: 0.639\n"
     ]
    }
   ],
   "source": [
    "print('y_true ', y_true['itemID'].tolist())\n",
    "print('y_pre  ', y_pre['itemID'].tolist())\n",
    "\n",
    "for i, top_k in enumerate(range(1, 7)):\n",
    "    # GroundTruth 中的 col_rating 在此不起任何作用  \n",
    "    eval_map = map_at_k(y_true, y_pre, col_user='userID', col_item='itemID', col_rating='userID', k=top_k)\n",
    "\n",
    "    print('TopK {} :{:.3f}   U1: {:.3f}  U2: {:.3f}  (U1+U2)/2: {:.3f}'.format(\n",
    "        top_k, eval_map, eval_1_map[i], eval_2_map[i], (eval_1_map[i]+eval_2_map[i])/2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "##  NDCG    \n",
    "Normalized Discounted Cumulative Gain(归一化折损累计增益)  \n",
    "\n",
    "https://www.cnblogs.com/by-dream/p/9403984.html   \n",
    "此脚本 相关性分数为 [0,1]， 若需要根据得分需要对应修改代码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 相关性分数为 [0,1]， 若需要根据得分需要对应修改代码\n",
    "\n",
    "def idcg(x, top_k):\n",
    "    return sum(1 / np.log1p(range(1, min(x, top_k) + 1)))\n",
    "\n",
    "def cg(x):\n",
    "    return 1 / np.log1p(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y1_true  [6, 2, 3, 4, 5, 1]\n",
      "y1_pre   [3, 20, 1]\n",
      "TopK 1 :1.000\n",
      "TopK 2 :0.613\n",
      "TopK 3 :0.704\n",
      "TopK 4 :0.586\n",
      "TopK 5 :0.509\n",
      "TopK 6 :0.454\n",
      "\n",
      "TopK 1 :1.000  1.000\n",
      "TopK 2 :0.613  0.613\n",
      "TopK 3 :0.704  0.704\n",
      "TopK 4 :0.586  0.586\n",
      "TopK 5 :0.509  0.509\n",
      "TopK 6 :0.454  0.454\n"
     ]
    }
   ],
   "source": [
    "print('y1_true ', y1_true['itemID'].tolist())\n",
    "print('y1_pre  ', y1_pre['itemID'].tolist())\n",
    "\n",
    "eval_1_ndcg = list()\n",
    "for top_k in range(1, 7):\n",
    "    # GroundTruth 中的 col_rating 在此不起任何作用  \n",
    "    eval_ndcg = ndcg_at_k(y1_true, y1_pre, col_user='userID', col_item='itemID', col_rating='userID', k=top_k)\n",
    "    eval_1_ndcg.append(eval_ndcg)\n",
    "    print('TopK {} :{:.3f}'.format(top_k, eval_ndcg))\n",
    "    \n",
    "print('')\n",
    "top_k = 1  # 预测结果 [3]  -》[T]\n",
    "print('TopK {} :{:.3f}  {:.3f}'.format(top_k, eval_1_ndcg[top_k-1], cg(1)/idcg(len(y1_true), top_k)))\n",
    "\n",
    "top_k = 2  # 预测结果 [3, 20]  -》[T, F]\n",
    "print('TopK {} :{:.3f}  {:.3f}'.format(top_k, eval_1_ndcg[top_k-1], cg(1)/idcg(len(y1_true), top_k)))\n",
    "\n",
    "top_k = 3  # 预测结果 [3, 20, 1]  -》[T, F, T]\n",
    "print('TopK {} :{:.3f}  {:.3f}'.format(top_k, eval_1_ndcg[top_k-1], (cg(1)+cg(3))/idcg(len(y1_true), top_k)))\n",
    "\n",
    "top_k = 4  # 预测结果 [3, 20, 1]  -》[T, F, T]\n",
    "print('TopK {} :{:.3f}  {:.3f}'.format(top_k, eval_1_ndcg[top_k-1], (cg(1)+cg(3))/idcg(len(y1_true), top_k)))\n",
    "\n",
    "top_k = 5  # 预测结果 [3, 20, 1]  -》[T, F, T]\n",
    "print('TopK {} :{:.3f}  {:.3f}'.format(top_k, eval_1_ndcg[top_k-1], (cg(1)+cg(3))/idcg(len(y1_true), top_k)))\n",
    "\n",
    "top_k = 6  # 预测结果 [3, 20, 1]  -》[T, F, T]\n",
    "print('TopK {} :{:.3f}  {:.3f}'.format(top_k, eval_1_ndcg[top_k-1], (cg(1)+cg(3))/idcg(len(y1_true), top_k)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y2_true  [5, 3, 1, 4]\n",
      "y2_pre   [3, 1, 4, 5]\n",
      "TopK 1 :1.000\n",
      "TopK 2 :1.000\n",
      "TopK 3 :1.000\n",
      "TopK 4 :1.000\n",
      "TopK 5 :1.000\n",
      "TopK 6 :1.000\n"
     ]
    }
   ],
   "source": [
    "print('y2_true ', y2_true['itemID'].tolist())\n",
    "print('y2_pre  ', y2_pre['itemID'].tolist())\n",
    "\n",
    "eval_2_ndcg = list()\n",
    "for top_k in range(1, 7):\n",
    "    # GroundTruth 中的 col_rating 在此不起任何作用  \n",
    "    eval_ndcg = ndcg_at_k(y2_true, y2_pre, col_user='userID', col_item='itemID', col_rating='userID', k=top_k)\n",
    "    eval_2_ndcg.append(eval_ndcg)\n",
    "    print('TopK {} :{:.3f}'.format(top_k, eval_ndcg))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_true  [6, 2, 3, 4, 5, 1, 5, 3, 1, 4]\n",
      "y_pre   [3, 20, 1, 3, 1, 4, 5]\n",
      "TopK 1 :1.000   U1: 1.000  U2: 1.000  (U1+U2)/2: 1.000\n",
      "TopK 2 :0.807   U1: 0.613  U2: 1.000  (U1+U2)/2: 0.807\n",
      "TopK 3 :0.852   U1: 0.704  U2: 1.000  (U1+U2)/2: 0.852\n",
      "TopK 4 :0.793   U1: 0.586  U2: 1.000  (U1+U2)/2: 0.793\n",
      "TopK 5 :0.754   U1: 0.509  U2: 1.000  (U1+U2)/2: 0.754\n",
      "TopK 6 :0.727   U1: 0.454  U2: 1.000  (U1+U2)/2: 0.727\n"
     ]
    }
   ],
   "source": [
    "print('y_true ', y_true['itemID'].tolist())\n",
    "print('y_pre  ', y_pre['itemID'].tolist())\n",
    "\n",
    "for i, top_k in enumerate(range(1, 7)):\n",
    "    # GroundTruth 中的 col_rating 在此不起任何作用  \n",
    "    eval_ndcg = ndcg_at_k(y_true, y_pre, col_user='userID', col_item='itemID', col_rating='userID', k=top_k)\n",
    "\n",
    "    print('TopK {} :{:.3f}   U1: {:.3f}  U2: {:.3f}  (U1+U2)/2: {:.3f}'.format(\n",
    "        top_k, eval_ndcg, eval_1_ndcg[i], eval_2_ndcg[i], (eval_1_ndcg[i]+eval_2_ndcg[i])/2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y1_true  [6, 2, 3, 4, 5, 1]\n",
      "y1_pre   [3, 20, 1]\n",
      "TopK 1 :1.000\n",
      "TopK 2 :0.500\n",
      "TopK 3 :0.667\n",
      "TopK 4 :0.500\n",
      "TopK 5 :0.400\n",
      "TopK 6 :0.333\n",
      "Top 1: 1/1 =1.000\n",
      "Top 2: 1/2 =0.500\n",
      "Top 3: 2/3 =0.667\n",
      "Top 4: 2/4 =0.500\n",
      "Top 5: 2/5 =0.400\n",
      "Top 6: 2/6 =0.333\n"
     ]
    }
   ],
   "source": [
    "print('y1_true ', y1_true['itemID'].tolist())\n",
    "print('y1_pre  ', y1_pre['itemID'].tolist())\n",
    "\n",
    "eval_1_precision = list()\n",
    "for top_k in range(1, 7):\n",
    "    # GroundTruth 中的 col_rating 在此不起任何作用  \n",
    "    eval_precision = precision_at_k(y1_true, y1_pre, col_user='userID', col_item='itemID', col_rating='userID', k=top_k)\n",
    "    eval_1_precision.append(eval_precision)\n",
    "    print('TopK {} :{:.3f}'.format(top_k, eval_precision))\n",
    "\n",
    "# max(len(y_pre), top_k)\n",
    "print('Top 1: 1/1 ={:.3f}'.format(1/1))\n",
    "print('Top 2: 1/2 ={:.3f}'.format(1/2))\n",
    "print('Top 3: 2/3 ={:.3f}'.format(2/3))\n",
    "print('Top 4: 2/4 ={:.3f}'.format(2/4))\n",
    "print('Top 5: 2/5 ={:.3f}'.format(2/5))\n",
    "print('Top 6: 2/6 ={:.3f}'.format(2/6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y2_true  [5, 3, 1, 4]\n",
      "y2_pre   [3, 1, 4, 5]\n",
      "TopK 1 :1.000\n",
      "TopK 2 :1.000\n",
      "TopK 3 :1.000\n",
      "TopK 4 :1.000\n",
      "TopK 5 :0.800\n",
      "TopK 6 :0.667\n",
      "Top 1: 1/1 =1.000\n",
      "Top 2: 2/2 =1.000\n",
      "Top 3: 3/3 =1.000\n",
      "Top 4: 4/4 =1.000\n",
      "Top 5: 4/5 =0.800\n",
      "Top 6: 4/6 =0.667\n"
     ]
    }
   ],
   "source": [
    "print('y2_true ', y2_true['itemID'].tolist())\n",
    "print('y2_pre  ', y2_pre['itemID'].tolist())\n",
    "\n",
    "eval_2_precision = list()\n",
    "for top_k in range(1, 7):\n",
    "    # GroundTruth 中的 col_rating 在此不起任何作用  \n",
    "    eval_precision = precision_at_k(y2_true, y2_pre, col_user='userID', col_item='itemID', col_rating='userID', k=top_k)\n",
    "    eval_2_precision.append(eval_precision)\n",
    "    print('TopK {} :{:.3f}'.format(top_k, eval_precision))\n",
    "\n",
    "# max(len(y_pre), top_k)\n",
    "print('Top 1: 1/1 ={:.3f}'.format(1/1))\n",
    "print('Top 2: 2/2 ={:.3f}'.format(2/2))\n",
    "print('Top 3: 3/3 ={:.3f}'.format(3/3))\n",
    "print('Top 4: 4/4 ={:.3f}'.format(4/4))\n",
    "print('Top 5: 4/5 ={:.3f}'.format(4/5))\n",
    "print('Top 6: 4/6 ={:.3f}'.format(4/6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_true  [6, 2, 3, 4, 5, 1, 5, 3, 1, 4]\n",
      "y_pre   [3, 20, 1, 3, 1, 4, 5]\n",
      "TopK 1 :1.000   U1: 1.000  U2: 1.000  (U1+U2)/2: 1.000\n",
      "TopK 2 :0.750   U1: 0.500  U2: 1.000  (U1+U2)/2: 0.750\n",
      "TopK 3 :0.833   U1: 0.667  U2: 1.000  (U1+U2)/2: 0.833\n",
      "TopK 4 :0.750   U1: 0.500  U2: 1.000  (U1+U2)/2: 0.750\n",
      "TopK 5 :0.600   U1: 0.400  U2: 0.800  (U1+U2)/2: 0.600\n",
      "TopK 6 :0.500   U1: 0.333  U2: 0.667  (U1+U2)/2: 0.500\n"
     ]
    }
   ],
   "source": [
    "print('y_true ', y_true['itemID'].tolist())\n",
    "print('y_pre  ', y_pre['itemID'].tolist())\n",
    "\n",
    "for i, top_k in enumerate(range(1, 7)):\n",
    "    # GroundTruth 中的 col_rating 在此不起任何作用  \n",
    "    eval_precision = precision_at_k(y_true, y_pre, col_user='userID', col_item='itemID', col_rating='userID', k=top_k)\n",
    "\n",
    "    print('TopK {} :{:.3f}   U1: {:.3f}  U2: {:.3f}  (U1+U2)/2: {:.3f}'.format(\n",
    "        top_k, eval_precision, eval_1_precision[i], eval_2_precision[i], (eval_1_precision[i]+eval_2_precision[i])/2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y1_true  [6, 2, 3, 4, 5, 1]\n",
      "y1_pre   [3, 20, 1]\n",
      "TopK 1 :0.167\n",
      "TopK 2 :0.167\n",
      "TopK 3 :0.333\n",
      "TopK 4 :0.333\n",
      "TopK 5 :0.333\n",
      "TopK 6 :0.333\n",
      "Top 1: 1/6 =0.167\n",
      "Top 2: 1/6 =0.167\n",
      "Top 3: 2/6 =0.333\n",
      "Top 4: 2/6 =0.333\n",
      "Top 5: 2/6 =0.333\n",
      "Top 6: 2/6 =0.333\n"
     ]
    }
   ],
   "source": [
    "print('y1_true ', y1_true['itemID'].tolist())\n",
    "print('y1_pre  ', y1_pre['itemID'].tolist())\n",
    "\n",
    "eval_1_recall = list()\n",
    "for top_k in range(1, 7):\n",
    "    # GroundTruth 中的 col_rating 在此不起任何作用  \n",
    "    eval_recall = recall_at_k(y1_true, y1_pre, col_user='userID', col_item='itemID', col_rating='userID', k=top_k)\n",
    "    eval_1_recall.append(eval_recall)\n",
    "    print('TopK {} :{:.3f}'.format(top_k, eval_recall))\n",
    "\n",
    "# 6 为 len(y1_true)\n",
    "print('Top 1: 1/6 ={:.3f}'.format(1/6))\n",
    "print('Top 2: 1/6 ={:.3f}'.format(1/6))\n",
    "print('Top 3: 2/6 ={:.3f}'.format(2/6))\n",
    "print('Top 4: 2/6 ={:.3f}'.format(2/6))\n",
    "print('Top 5: 2/6 ={:.3f}'.format(2/6))\n",
    "print('Top 6: 2/6 ={:.3f}'.format(2/6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y2_true  [5, 3, 1, 4]\n",
      "y2_pre   [3, 1, 4, 5]\n",
      "TopK 1 :0.250\n",
      "TopK 2 :0.500\n",
      "TopK 3 :0.750\n",
      "TopK 4 :1.000\n",
      "TopK 5 :1.000\n",
      "TopK 6 :1.000\n",
      "Top 1: 1/4 =0.250\n",
      "Top 2: 2/4 =0.500\n",
      "Top 3: 3/4 =0.750\n",
      "Top 4: 4/4 =1.000\n",
      "Top 5: 4/4 =1.000\n",
      "Top 6: 4/4 =1.000\n"
     ]
    }
   ],
   "source": [
    "print('y2_true ', y2_true['itemID'].tolist())\n",
    "print('y2_pre  ', y2_pre['itemID'].tolist())\n",
    "\n",
    "eval_2_recall = list()\n",
    "for top_k in range(1, 7):\n",
    "    # GroundTruth 中的 col_rating 在此不起任何作用  \n",
    "    eval_recall = recall_at_k(y2_true, y2_pre, col_user='userID', col_item='itemID', col_rating='userID', k=top_k)\n",
    "    eval_2_recall.append(eval_recall)\n",
    "    print('TopK {} :{:.3f}'.format(top_k, eval_recall))\n",
    "\n",
    "# 4 为 len(y2_true)\n",
    "print('Top 1: 1/4 ={:.3f}'.format(1/4))\n",
    "print('Top 2: 2/4 ={:.3f}'.format(2/4))\n",
    "print('Top 3: 3/4 ={:.3f}'.format(3/4))\n",
    "print('Top 4: 4/4 ={:.3f}'.format(4/4))\n",
    "print('Top 5: 4/4 ={:.3f}'.format(4/4))\n",
    "print('Top 6: 4/4 ={:.3f}'.format(4/4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_true  [6, 2, 3, 4, 5, 1, 5, 3, 1, 4]\n",
      "y_pre   [3, 20, 1, 3, 1, 4, 5]\n",
      "TopK 1 :0.208   U1: 0.167  U2: 0.250  (U1+U2)/2: 0.208\n",
      "TopK 2 :0.333   U1: 0.167  U2: 0.500  (U1+U2)/2: 0.333\n",
      "TopK 3 :0.542   U1: 0.333  U2: 0.750  (U1+U2)/2: 0.542\n",
      "TopK 4 :0.667   U1: 0.333  U2: 1.000  (U1+U2)/2: 0.667\n",
      "TopK 5 :0.667   U1: 0.333  U2: 1.000  (U1+U2)/2: 0.667\n",
      "TopK 6 :0.667   U1: 0.333  U2: 1.000  (U1+U2)/2: 0.667\n"
     ]
    }
   ],
   "source": [
    "print('y_true ', y_true['itemID'].tolist())\n",
    "print('y_pre  ', y_pre['itemID'].tolist())\n",
    "\n",
    "for i, top_k in enumerate(range(1, 7)):\n",
    "    # GroundTruth 中的 col_rating 在此不起任何作用  \n",
    "    eval_recall = recall_at_k(y_true, y_pre, col_user='userID', col_item='itemID', col_rating='userID', k=top_k)\n",
    "\n",
    "    print('TopK {} :{:.3f}   U1: {:.3f}  U2: {:.3f}  (U1+U2)/2: {:.3f}'.format(\n",
    "        top_k, eval_recall, eval_1_recall[i], eval_2_recall[i], (eval_1_recall[i]+eval_2_recall[i])/2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## RMSE     \n",
    "Root Mean Squared Error      \n",
    "均方根误差是预测值与真实值偏差的平方与观测次数n比值的平方根"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y1_true  [6, 2, 3, 4, 5, 1] [0.9, 0.5, 1.0, 1.0, 0.5, 0.5]\n",
      "y1_pre   [3, 20, 1] [1.0, 0.97, 0.5]\n",
      "rmse: 0.000\n",
      "\n",
      "   userID  itemID  rating  prediction\n",
      "0       1       3     1.0         1.0\n",
      "1       1       1     0.5         0.5\n",
      "rmse: 0.000\n"
     ]
    }
   ],
   "source": [
    "# 只考虑 匹配中 Items rating 和 prediction  (忽略未召回数据)\n",
    "\n",
    "print('y1_true ', y1_true['itemID'].tolist(), y1_true['rating'].tolist())\n",
    "print('y1_pre  ', y1_pre['itemID'].tolist(), y1_pre['prediction'].tolist())\n",
    "\n",
    "eval_1_mse = rmse(y1_true, y1_pre, col_user='userID', col_item='itemID', col_rating='rating')\n",
    "print('rmse: {:.3f}\\n'.format(eval_1_mse))\n",
    "\n",
    "y_true_pre = pd.merge(y1_true, y1_pre) \n",
    "eval_mse_2 = np.sqrt(mean_squared_error(y_true_pre['rating'].values, y_true_pre['prediction'].values))\n",
    "print(y_true_pre)\n",
    "print('rmse: {:.3f}'.format(eval_mse_2))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y2_true  [5, 3, 1, 4]\n",
      "y2_pre   [3, 1, 4, 5]\n",
      "rmse: 0.261\n",
      "\n",
      "   userID  itemID  rating  prediction\n",
      "0       2       5     1.0        0.92\n",
      "1       2       3     0.8        1.00\n",
      "2       2       1     0.9        0.97\n",
      "3       2       4     0.5        0.97\n",
      "rmse: 0.261\n"
     ]
    }
   ],
   "source": [
    "print('y2_true ', y2_true['itemID'].tolist())\n",
    "print('y2_pre  ', y2_pre['itemID'].tolist())\n",
    "\n",
    "eval_2_mse = rmse(y2_true, y2_pre, col_user='userID', col_item='itemID', col_rating='rating')\n",
    "print('rmse: {:.3f}\\n'.format(eval_2_mse))\n",
    "\n",
    "y_true_pre = pd.merge(y2_true, y2_pre) \n",
    "eval_mse_2 = np.sqrt(mean_squared_error(y_true_pre['rating'].values, y_true_pre['prediction'].values))\n",
    "print(y_true_pre)\n",
    "print('rmse: {:.3f}'.format(eval_mse_2)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_true  [6, 2, 3, 4, 5, 1, 5, 3, 1, 4]\n",
      "y_pre   [3, 20, 1, 3, 1, 4, 5]\n",
      "rmse: 0.213   U1: 0.000  U2: 0.261  (U1+U2)/2: 0.130 \n",
      "\n",
      "   userID  itemID  rating  prediction\n",
      "0       1       3     1.0        1.00\n",
      "1       1       1     0.5        0.50\n",
      "2       2       5     1.0        0.92\n",
      "3       2       3     0.8        1.00\n",
      "4       2       1     0.9        0.97\n",
      "5       2       4     0.5        0.97\n",
      "rmse: 0.213\n"
     ]
    }
   ],
   "source": [
    "print('y_true ', y_true['itemID'].tolist())\n",
    "print('y_pre  ', y_pre['itemID'].tolist())\n",
    "\n",
    "eval_mse = rmse(y_true, y_pre, col_user='userID', col_item='itemID', col_rating='rating')\n",
    "print('rmse: {:.3f}   U1: {:.3f}  U2: {:.3f}  (U1+U2)/2: {:.3f} \\n'.format(\n",
    "    eval_mse, eval_1_mse, eval_2_mse, (eval_1_mse+eval_2_mse)/2))  # 非直接平均\n",
    "\n",
    "\n",
    "y_true_pre = pd.merge(y_true, y_pre) \n",
    "eval_mse_2 = np.sqrt(mean_squared_error(y_true_pre['rating'].values, y_true_pre['prediction'].values))\n",
    "print(y_true_pre)\n",
    "print('rmse: {:.3f}'.format(eval_mse_2)) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## MAE      \n",
    "（Mean Absolute Error） 平均绝对误差 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y1_true  [6, 2, 3, 4, 5, 1] [0.9, 0.5, 1.0, 1.0, 0.5, 0.5]\n",
      "y1_pre   [3, 20, 1] [1.0, 0.97, 0.5]\n",
      "rmse: 0.000\n",
      "\n",
      "   userID  itemID  rating  prediction\n",
      "0       1       3     1.0         1.0\n",
      "1       1       1     0.5         0.5\n",
      "rmse: 0.000\n"
     ]
    }
   ],
   "source": [
    "# 只考虑 匹配中 Items rating 和 prediction  (忽略未召回数据)\n",
    "\n",
    "print('y1_true ', y1_true['itemID'].tolist(), y1_true['rating'].tolist())\n",
    "print('y1_pre  ', y1_pre['itemID'].tolist(), y1_pre['prediction'].tolist())\n",
    "\n",
    "eval_1_mae = mae(y1_true, y1_pre, col_user='userID', col_item='itemID', col_rating='rating')\n",
    "print('rmse: {:.3f}\\n'.format(eval_1_mae))\n",
    "\n",
    "y_true_pre = pd.merge(y1_true, y1_pre) \n",
    "eval_mae_2 = mean_absolute_error(y_true_pre['rating'].values, y_true_pre['prediction'].values)\n",
    "print(y_true_pre)\n",
    "print('rmse: {:.3f}'.format(eval_mae_2))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y2_true  [5, 3, 1, 4]\n",
      "y2_pre   [3, 1, 4, 5]\n",
      "rmse: 0.205\n",
      "\n",
      "   userID  itemID  rating  prediction\n",
      "0       2       5     1.0        0.92\n",
      "1       2       3     0.8        1.00\n",
      "2       2       1     0.9        0.97\n",
      "3       2       4     0.5        0.97\n",
      "rmse: 0.205\n"
     ]
    }
   ],
   "source": [
    "print('y2_true ', y2_true['itemID'].tolist())\n",
    "print('y2_pre  ', y2_pre['itemID'].tolist())\n",
    "\n",
    "eval_2_mae = mae(y2_true, y2_pre, col_user='userID', col_item='itemID', col_rating='rating')\n",
    "print('rmse: {:.3f}\\n'.format(eval_2_mae))\n",
    "\n",
    "y_true_pre = pd.merge(y2_true, y2_pre) \n",
    "eval_mae_2 = mean_absolute_error(y_true_pre['rating'].values, y_true_pre['prediction'].values)\n",
    "print(y_true_pre)\n",
    "print('rmse: {:.3f}'.format(eval_mae_2)) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_true  [6, 2, 3, 4, 5, 1, 5, 3, 1, 4]\n",
      "y_pre   [3, 20, 1, 3, 1, 4, 5]\n",
      "rmae: 0.137   U1: 0.000  U2: 0.205  (U1+U2)/2: 0.102 \n",
      "\n",
      "   userID  itemID  rating  prediction\n",
      "0       1       3     1.0        1.00\n",
      "1       1       1     0.5        0.50\n",
      "2       2       5     1.0        0.92\n",
      "3       2       3     0.8        1.00\n",
      "4       2       1     0.9        0.97\n",
      "5       2       4     0.5        0.97\n",
      "rmse: 0.137\n"
     ]
    }
   ],
   "source": [
    "print('y_true ', y_true['itemID'].tolist())\n",
    "print('y_pre  ', y_pre['itemID'].tolist())\n",
    "\n",
    "eval_mae = mae(y_true, y_pre, col_user='userID', col_item='itemID', col_rating='rating')\n",
    "print('rmae: {:.3f}   U1: {:.3f}  U2: {:.3f}  (U1+U2)/2: {:.3f} \\n'.format(\n",
    "    eval_mae, eval_1_mae, eval_2_mae, (eval_1_mae+eval_2_mae)/2))  # 非直接平均\n",
    "\n",
    "\n",
    "y_true_pre = pd.merge(y_true, y_pre) \n",
    "eval_mae_2 = mean_absolute_error(y_true_pre['rating'].values, y_true_pre['prediction'].values)\n",
    "print(y_true_pre)\n",
    "print('rmse: {:.3f}'.format(eval_mae_2)) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PY36-tf12",
   "language": "python",
   "name": "py36_b"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
